{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d1cefa5d",
   "metadata": {},
   "source": [
    "Q(state,action) = state下最优action分数 + 误差\n",
    "\n",
    "为了限制误差的范围,可以对它去均值,相当于去基线\n",
    "\n",
    "Q(state,action) = state下最优action分数 + 误差 - mean(误差)\n",
    "\n",
    "这么理解起来有点蛋疼,所以我这么理解\n",
    "\n",
    "Q(state,action) = state分数 + action分数 - mean(action分数)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "91162df8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAT5klEQVR4nO3db0xb570H8K8N2ITAMYEUu1ywwr3LXYLypytJ4LT3alPrhnaoahZ0tU1Ry6oovc1M1JQp2pDadMm2UmUvunVNyZst6V6kmZhuNhWl7Si0RFOd0NCiS6DhtnfJhZvEdhvEMdBgA/7dFxnn1ilJY/74sePvRzpSznke27/zGH9z/vgcW0REQESkgFV1AUSUvhhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkjLIAOnjwIFasWIHs7GxUVlaiq6tLVSlEpIiSAPrDH/6AhoYGPPfcc/jggw+wfv16VFdXIxgMqiiHiBSxqLgYtbKyEhs3bsTLL78MAIhGoygtLcWuXbvwk5/8JNHlEJEimYl+wUgkgu7ubjQ2NprLrFYrPB4PfD7frI8Jh8MIh8PmfDQaxfDwMAoLC2GxWBa9ZiKKj4hgdHQUxcXFsFpvvKOV8AD67LPPMD09DafTGbPc6XTi3Llzsz6mqakJ+/btS0R5RLSAhoaGUFJScsP2hAfQXDQ2NqKhocGcNwwDbrcbQ0ND0DRNYWVENJtQKITS0lLk5eXdtF/CA2j58uXIyMhAIBCIWR4IBOByuWZ9jN1uh91u/9JyTdMYQERJ7KsOkST8LJjNZkNFRQXa29vNZdFoFO3t7dB1PdHlEJFCSnbBGhoaUFdXhw0bNmDTpk341a9+hfHxcTz++OMqyiEiRZQE0He/+118+umn2Lt3L/x+P+666y68+eabXzowTUS3NyXfA5qvUCgEh8MBwzB4DIgoCd3qZ5TXghGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEiZuAPo5MmTePjhh1FcXAyLxYI//elPMe0igr179+LOO+/EkiVL4PF48PHHH8f0GR4exrZt26BpGvLz87F9+3aMjY3Na0WIKPXEHUDj4+NYv349Dh48OGv7gQMH8NJLL+HQoUM4ffo0li5diurqakxMTJh9tm3bhr6+PrS1taG1tRUnT57EE088Mfe1IKLUJPMAQI4fP27OR6NRcblc8stf/tJcNjIyIna7XV577TUREenv7xcA8v7775t93njjDbFYLHLx4sVbel3DMASAGIYxn/KJaJHc6md0QY8BnT9/Hn6/Hx6Px1zmcDhQWVkJn88HAPD5fMjPz8eGDRvMPh6PB1arFadPn571ecPhMEKhUMxERKlvQQPI7/cDAJxOZ8xyp9Nptvn9fhQVFcW0Z2ZmoqCgwOxzvaamJjgcDnMqLS1dyLKJSJGUOAvW2NgIwzDMaWhoSHVJRLQAFjSAXC4XACAQCMQsDwQCZpvL5UIwGIxpn5qawvDwsNnnena7HZqmxUxElPoWNIDKysrgcrnQ3t5uLguFQjh9+jR0XQcA6LqOkZERdHd3m306OjoQjUZRWVm5kOUQUZLLjPcBY2Nj+OSTT8z58+fPo6enBwUFBXC73di9ezd+/vOfY+XKlSgrK8Ozzz6L4uJibNmyBQCwevVqPPjgg9ixYwcOHTqEyclJ1NfX43vf+x6Ki4sXbMWIKAXEe3rtnXfeEQBfmurq6kTk2qn4Z599VpxOp9jtdrn//vtlYGAg5jmuXLki3//+9yU3N1c0TZPHH39cRkdHF/wUHxGpcaufUYuIiML8m5NQKASHwwHDMHg8iCgJ3epnNCXOghHR7YkBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESkT98/yEC0kEcGY/xNExobNZXZHEXKLyhRWRYnCACLl/P/5NkYufGjO37H6XxlAaYK7YKSYABJVXQQpwgAitQQQBlDaYgCRYsIASmMMIFJKRIDU+3FeWiAMIFKMW0DpjAFEagmAKAMoXcUVQE1NTdi4cSPy8vJQVFSELVu2YGBgIKbPxMQEvF4vCgsLkZubi9raWgQCgZg+g4ODqKmpQU5ODoqKirBnzx5MTU3Nf20oBcm13TBKS3EFUGdnJ7xeL06dOoW2tjZMTk5i8+bNGB8fN/s8/fTTeP3119HS0oLOzk5cunQJW7duNdunp6dRU1ODSCSC9957D6+++iqOHDmCvXv3LtxaUcq4dgyIW0BpS+YhGAwKAOns7BQRkZGREcnKypKWlhazz0cffSQAxOfziYjIiRMnxGq1it/vN/s0NzeLpmkSDodv6XUNwxAAYhjGfMqnJDAVuSp9//G8dB3aYU7nO3+vuiyap1v9jM7rGJBhGACAgoICAEB3dzcmJyfh8XjMPqtWrYLb7YbP5wMA+Hw+rF27Fk6n0+xTXV2NUCiEvr6+WV8nHA4jFArFTHSb4PeA0tqcAygajWL37t249957sWbNGgCA3++HzWZDfn5+TF+n0wm/32/2+WL4zLTPtM2mqakJDofDnEpLS+daNiUZkSgkOh270MJzI+lizu+01+vF2bNncezYsYWsZ1aNjY0wDMOchoaGFv01KTGikxMxF6ICwJJlxYqqoUSb08Wo9fX1aG1txcmTJ1FSUmIud7lciEQiGBkZidkKCgQCcLlcZp+urq6Y55s5SzbT53p2ux12u30upVKSExHIdafhrZlZiqqhRItrC0hEUF9fj+PHj6OjowNlZbFXLFdUVCArKwvt7e3msoGBAQwODkLXdQCAruvo7e1FMBg0+7S1tUHTNJSXl89nXeg2YbFkqC6BEiSuLSCv14ujR4/iz3/+M/Ly8sxjNg6HA0uWLIHD4cD27dvR0NCAgoICaJqGXbt2Qdd1VFVVAQA2b96M8vJyPProozhw4AD8fj+eeeYZeL1ebuUQAMBi5TGgdBFXADU3NwMAvvWtb8UsP3z4MH7wgx8AAF588UVYrVbU1tYiHA6juroar7zyitk3IyMDra2t2LlzJ3Rdx9KlS1FXV4f9+/fPb03oNmEBGEBpwyKSel9DDYVCcDgcMAwDmqapLofmITx6BWdb9iE6OfH3JRb80wP/joJ/vFtpXTQ/t/oZ5X81lHS4C5Y++E5T0rHwe0Bpg+80JR2LlWfB0gUDiJIPd8HSBt9pSjr8HlD6YABR0uFB6PTBd5oUu+5bIBbAYrGoKYUSjgFESl27DizlvopGC4QBRErxXkDpjQFEakmUG0BpjAFESnEXLL0xgEgp7oKlNwYQqcXfBEtrDCBSSiTK3wVLYwwgUur627FSemEAkVI8BpTeGECklkyDZ8HSFwOIlIpOT1+XP5a/T5QOGECkVNgIQKJT5nxmdi6ylvA2u+mCAURKXf+rqBZrBm9IlkYYQJRULBYLwKvh0wYDiJKLxcJ7QqcRvtOUVCwWK7eA0ggDiJKMhTckSyNx/TIqUbympqYwNjZ2w/arV6/GzEclCiM0ioyJqVn722w25OTkLGiNpA4DiBZVT08Ptm7diugNLrl4pKoMdZ7V5vzf/nYe/1al42pk9gB67LHH8Pzzzy9KrZR4DCBaVJFIBBcvXrxhAI0YhbgUXon/Df8z8jKGYQ234+Kli7ganj2ARkZGFrFaSrS4jgE1Nzdj3bp10DQNmqZB13W88cYbZvvExAS8Xi8KCwuRm5uL2tpaBAKBmOcYHBxETU0NcnJyUFRUhD179mBqavY/Nrr9BSIrcHbsXzA8+Q/4n4k16BvTMc3Lw9JGXAFUUlKCF154Ad3d3Thz5gzuu+8+PPLII+jr6wMAPP3003j99dfR0tKCzs5OXLp0CVu3bjUfPz09jZqaGkQiEbz33nt49dVXceTIEezdu3dh14pSxtXpPESR9fc5C0Yn8xHlpWFpI65dsIcffjhm/he/+AWam5tx6tQplJSU4Le//S2OHj2K++67DwBw+PBhrF69GqdOnUJVVRX+8pe/oL+/H2+//TacTifuuusu/OxnP8OPf/xj/PSnP4XNZlu4NaOUUJh1ETlWA59HNVgxhTvt/3XtPtGUFuZ8DGh6ehotLS0YHx+Hruvo7u7G5OQkPB6P2WfVqlVwu93w+XyoqqqCz+fD2rVr4XQ6zT7V1dXYuXMn+vr68I1vfCOuGs6dO4fc3Ny5rgIlwIULF256w7H/vtAHe8d+fBopxdIMA5aJjzF9k3sEDQ8Po7+/fzFKpQV0szOfXxR3APX29kLXdUxMTCA3NxfHjx9HeXk5enp6YLPZkJ+fH9Pf6XTC7/cDAPx+f0z4zLTPtN1IOBxGOBw250OhEADAMAweP0pyX/WH2H/hU/Rf+PSWny8SifBAdAoYHx+/pX5xB9DXv/519PT0wDAM/PGPf0RdXR06OzvjLjAeTU1N2Ldv35eWV1ZWQtN45XSys1gsC3bbVZfLhXvuuWdBnosWz8xGwleJ+5vQNpsNX/va11BRUYGmpiasX78ev/71r+FyuWb93ykQCMDlcgG49sdz/VmxmfmZPrNpbGyEYRjmNDQ0FG/ZRJSE5n0pRjQaRTgcRkVFBbKystDe3m62DQwMYHBwELquAwB0XUdvby+CwaDZp62tDZqmoby8/IavYbfbzVP/MxMRpb64dsEaGxvx0EMPwe12Y3R0FEePHsW7776Lt956Cw6HA9u3b0dDQwMKCgqgaRp27doFXddRVVUFANi8eTPKy8vx6KOP4sCBA/D7/XjmmWfg9Xpht9sXZQWJKHnFFUDBYBCPPfYYLl++DIfDgXXr1uGtt97CAw88AAB48cUXYbVaUVtbi3A4jOrqarzyyivm4zMyMtDa2oqdO3dC13UsXboUdXV12L9//8KuFSWNjIwMaJp2w29Cxys7O3tBnoeSg0VS8EeZQqEQHA4HDMPg7liSC4fDMbvc85Wbm4tly5Yt2PPR4rjVzyivBaNFZbfbUVpaqroMSlK8HxARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJSJlN1AXMhIgCAUCikuBIims3MZ3Pms3ojKRlAV65cAQCUlpYqroSIbmZ0dBQOh+OG7SkZQAUFBQCAwcHBm64cxQqFQigtLcXQ0BA0TVNdTkrgmM2NiGB0dBTFxcU37ZeSAWS1Xjt05XA4+EcxB5qmcdzixDGL361sHPAgNBEpwwAiImVSMoDsdjuee+452O121aWkFI5b/Dhmi8siX3WejIhokaTkFhAR3R4YQESkDAOIiJRhABGRMikZQAcPHsSKFSuQnZ2NyspKdHV1qS5JmaamJmzcuBF5eXkoKirCli1bMDAwENNnYmICXq8XhYWFyM3NRW1tLQKBQEyfwcFB1NTUICcnB0VFRdizZw+mpqYSuSrKvPDCC7BYLNi9e7e5jGOWIJJijh07JjabTX73u99JX1+f7NixQ/Lz8yUQCKguTYnq6mo5fPiwnD17Vnp6euTb3/62uN1uGRsbM/s8+eSTUlpaKu3t7XLmzBmpqqqSe+65x2yfmpqSNWvWiMfjkQ8//FBOnDghy5cvl8bGRhWrlFBdXV2yYsUKWbdunTz11FPmco5ZYqRcAG3atEm8Xq85Pz09LcXFxdLU1KSwquQRDAYFgHR2doqIyMjIiGRlZUlLS4vZ56OPPhIA4vP5RETkxIkTYrVaxe/3m32am5tF0zQJh8OJXYEEGh0dlZUrV0pbW5t885vfNAOIY5Y4KbULFolE0N3dDY/HYy6zWq3weDzw+XwKK0sehmEA+P8Ldru7uzE5ORkzZqtWrYLb7TbHzOfzYe3atXA6nWaf6upqhEIh9PX1JbD6xPJ6vaipqYkZG4BjlkgpdTHqZ599hunp6Zg3HQCcTifOnTunqKrkEY1GsXv3btx7771Ys2YNAMDv98NmsyE/Pz+mr9PphN/vN/vMNqYzbbejY8eO4YMPPsD777//pTaOWeKkVADRzXm9Xpw9exZ//etfVZeS1IaGhvDUU0+hra0N2dnZqstJaym1C7Z8+XJkZGR86WxEIBCAy+VSVFVyqK+vR2trK9555x2UlJSYy10uFyKRCEZGRmL6f3HMXC7XrGM603a76e7uRjAYxN13343MzExkZmais7MTL730EjIzM+F0OjlmCZJSAWSz2VBRUYH29nZzWTQaRXt7O3RdV1iZOiKC+vp6HD9+HB0dHSgrK4tpr6ioQFZWVsyYDQwMYHBw0BwzXdfR29uLYDBo9mlra4OmaSgvL0/MiiTQ/fffj97eXvT09JjThg0bsG3bNvPfHLMEUX0UPF7Hjh0Tu90uR44ckf7+fnniiSckPz8/5mxEOtm5c6c4HA5599135fLly+b0+eefm32efPJJcbvd0tHRIWfOnBFd10XXdbN95pTy5s2bpaenR958802544470uqU8hfPgolwzBIl5QJIROQ3v/mNuN1usdlssmnTJjl16pTqkpQBMOt0+PBhs8/Vq1flhz/8oSxbtkxycnLkO9/5jly+fDnmeS5cuCAPPfSQLFmyRJYvXy4/+tGPZHJyMsFro871AcQxSwzejoOIlEmpY0BEdHthABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMr8H4dus3mDRoFyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        over = terminated or truncated\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            over = True\n",
    "        \n",
    "        #没坚持到最后,扣分\n",
    "        if over and self.step_n < 200:\n",
    "            reward = -1000\n",
    "\n",
    "        return state, reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "378392ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 2])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "class Model(torch.nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "        self.fc = torch.nn.Sequential(\n",
    "            torch.nn.Linear(4, 64),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(64, 64),\n",
    "            torch.nn.ReLU(),\n",
    "        )\n",
    "\n",
    "        self.fc_action = torch.nn.Linear(64, 2)\n",
    "        self.fc_state = torch.nn.Linear(64, 1)\n",
    "\n",
    "    def forward(self, state):\n",
    "        state = self.fc(state)\n",
    "\n",
    "        #评估state的价值\n",
    "        value_state = self.fc_state(state)\n",
    "\n",
    "        #每个state下每个action的价值\n",
    "        value_action = self.fc_action(state)\n",
    "\n",
    "        #综合以上两者计算最终的价值,action去均值是为了数值稳定\n",
    "        return value_state + value_action - value_action.mean(dim=-1,\n",
    "                                                              keepdim=True)\n",
    "\n",
    "\n",
    "model = Model()\n",
    "model_delay = Model()\n",
    "\n",
    "#复制参数\n",
    "model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "model(torch.randn(64, 4)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84cbf0ff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-993.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    data = []\n",
    "    reward_sum = 0\n",
    "\n",
    "    state = env.reset()\n",
    "    over = False\n",
    "    while not over:\n",
    "        action = model(torch.FloatTensor(state).reshape(1, 4)).argmax().item()\n",
    "        if random.random() < 0.1:\n",
    "            action = env.action_space.sample()\n",
    "\n",
    "        next_state, reward, over = env.step(action)\n",
    "\n",
    "        data.append((state, action, reward, next_state, over))\n",
    "        reward_sum += reward\n",
    "\n",
    "        state = next_state\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    return data, reward_sum\n",
    "\n",
    "\n",
    "play()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7bfcfd21",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_7125/2282743005.py:27: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(203,\n",
       " (array([-0.02135725, -0.00366478, -0.00759902, -0.04133787], dtype=float32),\n",
       "  1,\n",
       "  1.0,\n",
       "  array([-0.02143055,  0.1915653 , -0.00842578, -0.33640864], dtype=float32),\n",
       "  False))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据池\n",
    "class Pool:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.pool = []\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.pool)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return self.pool[i]\n",
    "\n",
    "    #更新动作池\n",
    "    def update(self):\n",
    "        #每次更新不少于N条新数据\n",
    "        old_len = len(self.pool)\n",
    "        while len(pool) - old_len < 200:\n",
    "            self.pool.extend(play()[0])\n",
    "\n",
    "        #只保留最新的N条数据\n",
    "        self.pool = self.pool[-2_0000:]\n",
    "\n",
    "    #获取一批数据样本\n",
    "    def sample(self):\n",
    "        data = random.sample(self.pool, 64)\n",
    "\n",
    "        state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n",
    "        action = torch.LongTensor([i[1] for i in data]).reshape(-1, 1)\n",
    "        reward = torch.FloatTensor([i[2] for i in data]).reshape(-1, 1)\n",
    "        next_state = torch.FloatTensor([i[3] for i in data]).reshape(-1, 4)\n",
    "        over = torch.LongTensor([i[4] for i in data]).reshape(-1, 1)\n",
    "\n",
    "        return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "pool = Pool()\n",
    "pool.update()\n",
    "pool.sample()\n",
    "\n",
    "len(pool), pool[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "018c4b1e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 403 -964.05\n",
      "100 20000 149.3\n",
      "200 20000 200.0\n",
      "300 20000 200.0\n",
      "400 20000 200.0\n",
      "500 20000 200.0\n",
      "600 20000 200.0\n",
      "700 20000 200.0\n",
      "800 20000 200.0\n",
      "900 20000 200.0\n"
     ]
    }
   ],
   "source": [
    "#训练\n",
    "def train():\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=2e-4)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #共更新N轮数据\n",
    "    for epoch in range(1000):\n",
    "        pool.update()\n",
    "\n",
    "        #每次更新数据后,训练N次\n",
    "        for i in range(200):\n",
    "\n",
    "            #采样N条数据\n",
    "            state, action, reward, next_state, over = pool.sample()\n",
    "\n",
    "            #计算value\n",
    "            value = model(state).gather(dim=1, index=action)\n",
    "\n",
    "            #计算target\n",
    "            with torch.no_grad():\n",
    "                target = model_delay(next_state)\n",
    "            target = target.max(dim=1)[0].reshape(-1, 1)\n",
    "            target = target * 0.99 * (1 - over) + reward\n",
    "\n",
    "            loss = loss_fn(value, target)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "        #复制参数\n",
    "        if (epoch + 1) % 5 == 0:\n",
    "            model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "        if epoch % 100 == 0:\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "            print(epoch, len(pool), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a101d0e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAATyUlEQVR4nO3dfVBbVZ8H8G8CJC2FGwpIIgvZ4ujYMn1T2sLV2dXRWFSmY5U/fOkoOp061tARcbuKo/V9cOrOo49a6T/aun/UdnCmOjL1BUNLxzEUi7JLqWV1n7rwtE1irdwALSEkZ//w4WpaqARoThK+n5k703vOSfK7J8m39yUhBiGEABGRBEbZBRDR7MUAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaaQF0LZt27BgwQLMmTMHpaWlaG9vl1UKEUkiJYD27NmD2tpaPPfcc/j222+xbNkylJeXw+fzySiHiCQxyPgyamlpKVauXIm3334bABAOh1FYWIhNmzbhqaeeinU5RCRJaqwfcGRkBB0dHairq9PbjEYjHA4H3G73uLcJBAIIBAL6ejgcxpkzZ5CTkwODwXDJayai6AghMDAwgPz8fBiNEx9oxTyATp8+jVAoBKvVGtFutVpx7NixcW9TX1+PF154IRblEdEM6uvrQ0FBwYT9MQ+gqairq0Ntba2+rmka7HY7+vr6oCiKxMqIaDx+vx+FhYXIzMy86LiYB1Bubi5SUlLg9Xoj2r1eL2w227i3MZvNMJvNF7QrisIAIopjf3aKJOZXwUwmE0pKSuByufS2cDgMl8sFVVVjXQ4RSSTlEKy2thZVVVVYsWIFVq1ahTfeeANDQ0N46KGHZJRDRJJICaC7774bP//8M7Zs2QKPx4Ply5fjs88+u+DENBElNymfA5ouv98Pi8UCTdN4DogoDk32PcrvghGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEiaqAPo4MGDWLNmDfLz82EwGPDRRx9F9AshsGXLFlx++eWYO3cuHA4Hfvjhh4gxZ86cwbp166AoCrKysrB+/XoMDg5Oa0OIKPFEHUBDQ0NYtmwZtm3bNm7/1q1b8eabb2L79u04dOgQ5s2bh/LycgwPD+tj1q1bh+7ubjQ3N6OpqQkHDx7Eww8/PPWtIKLEJKYBgNi7d6++Hg6Hhc1mE6+99pre1t/fL8xms/jggw+EEEIcPXpUABDffPONPubTTz8VBoNBnDhxYlKPq2maACA0TZtO+UR0iUz2PTqj54COHz8Oj8cDh8Oht1ksFpSWlsLtdgMA3G43srKysGLFCn2Mw+GA0WjEoUOHxr3fQCAAv98fsRBR4pvRAPJ4PAAAq9Ua0W61WvU+j8eDvLy8iP7U1FRkZ2frY85XX18Pi8WiL4WFhTNZNhFJkhBXwerq6qBpmr709fXJLomIZsCMBpDNZgMAeL3eiHav16v32Ww2+Hy+iP7R0VGcOXNGH3M+s9kMRVEiFiJKfDMaQEVFRbDZbHC5XHqb3+/HoUOHoKoqAEBVVfT396Ojo0Mf09LSgnA4jNLS0pksh4jiXGq0NxgcHMSPP/6orx8/fhydnZ3Izs6G3W5HTU0NXn75ZVx11VUoKirCs88+i/z8fKxduxYAsGjRItx6663YsGEDtm/fjmAwiOrqatxzzz3Iz8+fsQ0jogQQ7eW1/fv3CwAXLFVVVUKI3y7FP/vss8JqtQqz2Sxuvvlm0dPTE3Efv/zyi7j33ntFRkaGUBRFPPTQQ2JgYGDGL/ERkRyTfY8ahBBCYv5Nid/vh8VigaZpPB9EFIcm+x5NiKtgRJScGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNFH/LA9RPBPhMLS+IwiNnNPb5uUVYY4l7yK3IlkYQJRURHgUfW0fYrjfo7f987+sYwDFKR6CUVIRQiABf2lq1mIAUXIRYYABlDAYQJRUftv7YQAlCgYQJRcR5iFYAmEAUVIRQvAQLIEwgCi5iDB4CJY4ogqg+vp6rFy5EpmZmcjLy8PatWvR09MTMWZ4eBhOpxM5OTnIyMhAZWUlvF5vxJje3l5UVFQgPT0deXl52Lx5M0ZHR6e/NTTr8SpYYokqgFpbW+F0OtHW1obm5mYEg0GsXr0aQ0ND+pjHH38cn3zyCRobG9Ha2oqTJ0/irrvu0vtDoRAqKiowMjKCr7/+Gu+//z527tyJLVu2zNxW0ezFq2CJRUyDz+cTAERra6sQQoj+/n6RlpYmGhsb9THff/+9ACDcbrcQQoh9+/YJo9EoPB6PPqahoUEoiiICgcCkHlfTNAFAaJo2nfIpCQ37T4vv/vPfRPv2Dfri7T4gu6xZZ7Lv0WmdA9I0DQCQnZ0NAOjo6EAwGITD4dDHLFy4EHa7HW63GwDgdruxZMkSWK1WfUx5eTn8fj+6u7vHfZxAIAC/3x+xEI2Lh2AJZcoBFA6HUVNTg+uvvx6LFy8GAHg8HphMJmRlZUWMtVqt8Hg8+pg/hs9Y/1jfeOrr62GxWPSlsLBwqmVTkhPh0D9ORP/OYOC1lng15WfG6XTiyJEj2L1790zWM666ujpomqYvfX19l/wxKTGNnO3HaOD3c5KGlDSYlcskVkQXM6Uvo1ZXV6OpqQkHDx5EQUGB3m6z2TAyMoL+/v6IvSCv1wubzaaPaW9vj7i/satkY2POZzabYTabp1IqzTbnfQ7IYDDAkJImsSC6mKj2gIQQqK6uxt69e9HS0oKioqKI/pKSEqSlpcHlcultPT096O3thaqqAABVVdHV1QWfz6ePaW5uhqIoKC4uns62EI3DAIPBILsImkBUe0BOpxO7du3Cxx9/jMzMTP2cjcViwdy5c2GxWLB+/XrU1tYiOzsbiqJg06ZNUFUVZWVlAIDVq1ejuLgY999/P7Zu3QqPx4NnnnkGTqeTezk08wyAwZgiuwqaQFQB1NDQAAC48cYbI9p37NiBBx98EADw+uuvw2g0orKyEoFAAOXl5XjnnXf0sSkpKWhqasLGjRuhqirmzZuHqqoqvPjii9PbEqJxGQCehI5bBpGA1yz9fj8sFgs0TYOiKLLLoTjiP3EMPU1/0deNaXOw6I5/R3pOwUVuRTNtsu9R/tdASc1gMMBg5Ms8XvGZoeTHQ7C4xWeGkpyBH0SMY3xmKLkZwEOwOMZnhpIc94DiGZ8ZSn4MoLjFZ4aSijj/i6gAPwkdxxhAlFzC4T8fQ3GDAURJ5fw9IIpvDCBKKiIckl0CRYEBREmFe0CJhQFESYV7QImFAURJRfAkdEJhAFFS4SFYYmEAUXLhIVhCYQBRUuEhWGJhAFFSEYJ7QImEAURJRYRGIxsMBvz2hQyKRwwgSirnfj0ZsW7OzIUx1SSpGvozDCBKKuHz9oAMKWn/2AuieMQAoqRmMBr5bfg4xgCi5MbwiWsMIEpqv/01RIZQvGIAUVIzGIzcC4pjUf0yKpFsgUAA586dm7A/GAxGrI+GQtA0bcKfZ547dy5/ElwiBhAllD179uDpp5+esL9m7TL86+J/0tddLfvxH86/IBQe/weAt27divvuu2/G66TJYQBRQhkcHMSJEycm7j+7CH87txyngwXITevDwND/4e8nTiA8QQANDQ1dqlJpEqI6B9TQ0IClS5dCURQoigJVVfHpp5/q/cPDw3A6ncjJyUFGRgYqKyvh9Xoj7qO3txcVFRVIT09HXl4eNm/ejNHR0fMfimhKjp9biv85uwJngvn44exK/O/QYggxfviQfFEFUEFBAV599VV0dHTg8OHDuOmmm3DHHXegu7sbAPD444/jk08+QWNjI1pbW3Hy5Encdddd+u1DoRAqKiowMjKCr7/+Gu+//z527tyJLVu2zOxW0aw1OJqFsZe1gBH+oAVg/sStqA7B1qxZE7H+yiuvoKGhAW1tbSgoKMC7776LXbt24aabbgIA7NixA4sWLUJbWxvKysrwxRdf4OjRo/jyyy9htVqxfPlyvPTSS3jyySfx/PPPw2TiR+Zpei43/w1phmEEhRlphmHkpf3I/IljUz4HFAqF0NjYiKGhIaiqio6ODgSDQTgcDn3MwoULYbfb4Xa7UVZWBrfbjSVLlsBqtepjysvLsXHjRnR3d+Oaa66JqoZjx44hIyNjqptACcjj8Vy0/7/+uwV/95zAr0Er5qd5cdrbc9Hxp06dwtGjR2eyRMJv5+omI+oA6urqgqqqGB4eRkZGBvbu3Yvi4mJ0dnbCZDIhKysrYrzVatVfNB6PJyJ8xvrH+iYSCAQQCAT0db/fDwDQNI3nj2aZi12CB4CvunoB9EZ1f/39/dMrii4w2ZP7UQfQ1Vdfjc7OTmiahg8//BBVVVVobW2NusBo1NfX44UXXrigvbS0FIqiXNLHpvjS2dk5o/d3xRVX4LrrrpvR+6TfdxL+TNSfhDaZTLjyyitRUlKC+vp6LFu2DH/9619hs9kwMjJywf8mXq8XNpsNAGCz2S64Kja2PjZmPHV1ddA0TV/6+vqiLZuI4tC0v4oRDocRCARQUlKCtLQ0uFwuva+npwe9vb1QVRUAoKoqurq64PP59DHNzc1QFAXFxcUTPobZbNYv/Y8tRJT4ojoEq6urw2233Qa73Y6BgQHs2rULBw4cwOeffw6LxYL169ejtrYW2dnZUBQFmzZtgqqqKCsrAwCsXr0axcXFuP/++7F161Z4PB4888wzcDqd/Dg80SwUVQD5fD488MADOHXqFCwWC5YuXYrPP/8ct9xyCwDg9ddfh9FoRGVlJQKBAMrLy/HOO+/ot09JSUFTUxM2btwIVVUxb948VFVV4cUXX5zZraKkZTKZZnQPmB/9kMsgEvBjon6/HxaLBZqm8XBslhkcHMSvv/46Y/c3f/58fpTjEpjse5TfBaOEkpGRwcBIIvx7QEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEiaVNkFTIUQAgDg9/slV0JE4xl7b469VyeSkAH0yy+/AAAKCwslV0JEFzMwMACLxTJhf0IGUHZ2NgCgt7f3ohtHkfx+PwoLC9HX1wdFUWSXkxA4Z1MjhMDAwADy8/MvOi4hA8ho/O3UlcVi4YtiChRF4bxFiXMWvcnsHPAkNBFJwwAiImkSMoDMZjOee+45mM1m2aUkFM5b9Dhnl5ZB/Nl1MiKiSyQh94CIKDkwgIhIGgYQEUnDACIiaRIygLZt24YFCxZgzpw5KC0tRXt7u+ySpKmvr8fKlSuRmZmJvLw8rF27Fj09PRFjhoeH4XQ6kZOTg4yMDFRWVsLr9UaM6e3tRUVFBdLT05GXl4fNmzdjdHQ0lpsizauvvgqDwYCamhq9jXMWIyLB7N69W5hMJvHee++J7u5usWHDBpGVlSW8Xq/s0qQoLy8XO3bsEEeOHBGdnZ3i9ttvF3a7XQwODupjHnnkEVFYWChcLpc4fPiwKCsrE9ddd53ePzo6KhYvXiwcDof47rvvxL59+0Rubq6oq6uTsUkx1d7eLhYsWCCWLl0qHnvsMb2dcxYbCRdAq1atEk6nU18PhUIiPz9f1NfXS6wqfvh8PgFAtLa2CiGE6O/vF2lpaaKxsVEf8/333wsAwu12CyGE2LdvnzAajcLj8ehjGhoahKIoIhAIxHYDYmhgYEBcddVVorm5Wdxwww16AHHOYiehDsFGRkbQ0dEBh8OhtxmNRjgcDrjdbomVxQ9N0wD8/oXdjo4OBIPBiDlbuHAh7Ha7PmdutxtLliyB1WrVx5SXl8Pv96O7uzuG1ceW0+lERUVFxNwAnLNYSqgvo54+fRqhUCjiSQcAq9WKY8eOSaoqfoTDYdTU1OD666/H4sWLAQAejwcmkwlZWVkRY61WKzwejz5mvDkd60tGu3fvxrfffotvvvnmgj7OWewkVADRxTmdThw5cgRfffWV7FLiWl9fHx577DE0Nzdjzpw5ssuZ1RLqECw3NxcpKSkXXI3wer2w2WySqooP1dXVaGpqwv79+1FQUKC322w2jIyMoL+/P2L8H+fMZrONO6djfcmmo6MDPp8P1157LVJTU5GamorW1la8+eabSE1NhdVq5ZzFSEIFkMlkQklJCVwul94WDofhcrmgqqrEyuQRQqC6uhp79+5FS0sLioqKIvpLSkqQlpYWMWc9PT3o7e3V50xVVXR1dcHn8+ljmpuboSgKiouLY7MhMXTzzTejq6sLnZ2d+rJixQqsW7dO/zfnLEZknwWP1u7du4XZbBY7d+4UR48eFQ8//LDIysqKuBoxm2zcuFFYLBZx4MABcerUKX05e/asPuaRRx4RdrtdtLS0iMOHDwtVVYWqqnr/2CXl1atXi87OTvHZZ5+Jyy67bFZdUv7jVTAhOGexknABJIQQb731lrDb7cJkMolVq1aJtrY22SVJA2DcZceOHfqYc+fOiUcffVTMnz9fpKenizvvvFOcOnUq4n5++ukncdttt4m5c+eK3Nxc8cQTT4hgMBjjrZHn/ADinMUG/xwHEUmTUOeAiCi5MICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJp/h+2s54XFxDb6wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
